import inspect
from datetime import datetime, timedelta
import hashlib
from master.mongo_operator import MongoOperator
import task_settings
import json
import time
from utils.redis_helper import RedisHelper
import utils.general_settings as general_settings

if __name__ == '__main__':
    seconds_span = 300
    # 接受worker传递到redis队列的结果斌存入数据库（queue：response）
    print("mongodb_host : " + str(task_settings.mongodb_host), "\t\t\tmongodb_port : " + str(task_settings.mongodb_port))
    mongo_operator = MongoOperator(host=task_settings.mongodb_host, port=task_settings.mongodb_port)
    print("message_queue_host : " + str(general_settings.redis_host), "\t\t\tmessage_queue_port : " + str(general_settings.redis_port))
    mq_client = RedisHelper(host=general_settings.redis_host, port=general_settings.redis_port)
    print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\t\tmaster开始执行....................")
    last_time = datetime.now() + timedelta(seconds=seconds_span)
    # 创建出错任务的{md5：次数}字典，当循环次数超过n次时此任务丢弃
    error_task_dict = {}
    data_count = 0

    while True:
        try:
            if mq_client.get_list_length(general_settings.error_task_queue) > 0:
                command = mq_client.read_list(general_settings.error_task_queue)
                # 计算command的MD5值，用于验证是否是同一个command
                # 此句每次update前必须重新调用，否则会导致错误，比如同一字符串不一样结果
                hl = hashlib.md5()
                hl.update(str(command).encode(encoding='utf-8'))
                md5 = hl.hexdigest()
                # 将MD5值和出错任务字典中的key比较，存在则加1，不存在则创建并赋值为1,
                # 超过一定次数则抛弃该任务，否则返回任务队列再次执行
                if md5 in error_task_dict.keys():
                    error_task_dict[md5] = int(error_task_dict[md5]) + 1
                    # 循环次数超过 10 次，删除条目，不超过 10 次，返回task队列
                    if int(error_task_dict[md5]) > general_settings.error_task_loop_count:
                        error_task_dict.pop(md5)
                        print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\t\t删除任务：" + command)
                    else:
                        mq_client.write_list(command['queue'], command)
                        print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\t\t返回任务：" + command)
                else:
                    error_task_dict[md5] = 1
                    mq_client.write_list(command['queue'], command)
                    print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\t\t返回任务：" + str(json.loads(command)))

            if mq_client.get_list_length(general_settings.response_queue) > 0:
                response = mq_client.read_list(general_settings.response_queue)
                try:
                    mongo_operator.insert(response['db'], response['collection'], response['data'])
                    # print(response)
                    data_count += 1
                except Exception as e:
                    mq_client.write_list(general_settings.response_queue, response)
                    time.sleep(30)
            else:
                time.sleep(5)
            # 每300秒输出一次统计数据
            if (datetime.now() - last_time).seconds >= seconds_span:
                print(str.format(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\t\t已获取【{0}】条response，剩余【{1}】条",
                                 str(data_count), str(mq_client.get_list_length(general_settings.response_queue))))

                task_queue_list = []
                # 获取全部的配置类，并提取其中的队列名组成列表供检查队列是否有任务
                configs = inspect.getmembers(general_settings, inspect.isclass)
                for (name, _) in configs:
                    config = getattr(general_settings, name)
                    task_queue_list.append(config().urgent_task_queue)
                    task_queue_list.append(config().task_queue)

                print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\t\t任务队列剩余任务如下：")
                for task_queue in task_queue_list:
                    print(str.format("队列：{0}，剩余任务：{1}", task_queue, str(mq_client.get_list_length(task_queue))))
                last_time = datetime.now()
        except Exception as e:
            print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "master出错")
            print(e)
            time.sleep(60)
